#!/usr/bin/env python
# -*- coding: UTF-8 -*-
'''
@File    ：sms_un_base_v1.py
@IDE     ：PyCharm 
@Author  ：lmy
@Date    ：2024/8/6 20:04 
'''

from feature_set.sms.utils.data_utils import *
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import math


class SmsUnBaseV1:
    @staticmethod
    def extract_cnt_feature(data):
        """
        cnt模块:计算基础统计单位的数量以及占比
        :return: dict
        """
        data_drop_carrier = data[data['is_carrier'] == 0]
        data_eng_sender = data[data['is_digit'] == 0]
        data_carrier = data[data['is_carrier'] == 1]
        data_dig_sender = data[data['is_digit'] == 1]

        sms_feature_res = {}
        data_cnt = calc_cnt_fea(data)
        # 周末工作日
        time_type = 'date_period'
        for t in WEEK_TYPES:
            drop_carrier_df = data_drop_carrier[data_drop_carrier[time_type] == t]  # 去除运营商的数据
            eng_sender_df = data_eng_sender[data_eng_sender[time_type] == t]  # sender为字母的数据
            carrier_df = data_carrier[data_carrier[time_type] == t]  # 运营商的数据
            dig_sender_df = data_dig_sender[data_dig_sender[time_type] == t]  # sender为数字的数据
            all_df = data[data[time_type] == t]
            sms_feature_res = gen_cnt_fea(all_df, drop_carrier_df, 'drop_carrier_' + t, sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, eng_sender_df, 'eng_sender_' + t, sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, carrier_df, 'carrier_' + t, sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, dig_sender_df, 'dig_sender_' + t, sms_feature_res)

            all_cnt = calc_cnt_fea(all_df)
            for fea in all_cnt:
                sms_feature_res[f"{fea}_{t}"] = all_cnt[fea]
                sms_feature_res[f"{fea}_rto_{t}"] = all_cnt[fea] / data_cnt[fea] if data_cnt[fea] > 0 else -999

        # 不同时间窗口
        time_type = 'time_diff'
        for t in TIME_INTERVALS:
            if t == 'all':
                drop_carrier_df = data_drop_carrier.copy()  # 去除运营商的数据
                eng_sender_df = data_eng_sender.copy()  # sender为字母的数据
                carrier_df = data_carrier.copy()  # 运营商的数据
                dig_sender_df = data_dig_sender.copy()  # sender为数字的数据
                all_df = data.copy()
            else:
                drop_carrier_df = data_drop_carrier[data_drop_carrier[time_type] < t]  # 去除运营商的数据
                eng_sender_df = data_eng_sender[data_eng_sender[time_type] < t]  # sender为字母的数据
                carrier_df = data_carrier[data_carrier[time_type] < t]  # 运营商的数据
                dig_sender_df = data_dig_sender[data_dig_sender[time_type] < t]  # sender为数字的数据
                all_df = data[data[time_type] < t]
                t = 'd' + str(t)
            sms_feature_res = gen_cnt_fea(all_df, drop_carrier_df, 'drop_carrier_' + t, sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, eng_sender_df, 'eng_sender_' + t, sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, carrier_df, 'carrier_' + t, sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, dig_sender_df, 'dig_sender_' + t, sms_feature_res)

            all_cnt = calc_cnt_fea(all_df)
            for fea in all_cnt:
                sms_feature_res[f"{fea}_{t}"] = all_cnt[fea]
                sms_feature_res[f"{fea}_rto_{t}"] = all_cnt[fea] / data_cnt[fea] if data_cnt[fea] > 0 else -999

        # 每天不同时间段
        time_type = 'hour'
        for t in TIME_PERIODS:
            drop_carrier_df = data_drop_carrier[
                (data_drop_carrier[time_type] >= t[0]) & (data_drop_carrier[time_type] <= t[1])]
            eng_sender_df = data_eng_sender[
                (data_eng_sender[time_type] >= t[0]) & (data_eng_sender[time_type] <= t[1])]  # sender为字母的数据
            carrier_df = data_carrier[
                (data_carrier[time_type] >= t[0]) & (data_carrier[time_type] <= t[1])]
            dig_sender_df = data_dig_sender[
                (data_dig_sender[time_type] >= t[0]) & (data_dig_sender[time_type] <= t[1])]  # sender为字母的数据

            all_df = data[(data[time_type] >= t[0]) & (data[time_type] <= t[1])]

            sms_feature_res = gen_cnt_fea(all_df, drop_carrier_df, 'drop_carrier_' + t[-1], sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, eng_sender_df, 'eng_sender_' + t[-1], sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, carrier_df, 'carrier_' + t[-1], sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, dig_sender_df, 'dig_sender_' + t[-1], sms_feature_res)

            all_cnt = calc_cnt_fea(all_df)
            for fea in all_cnt:
                sms_feature_res[f"{fea}_{t[-1]}"] = all_cnt[fea]
                sms_feature_res[f"{fea}_rto_{t[-1]}"] = all_cnt[fea] / data_cnt[fea] if data_cnt[fea] > 0 else -999

        # 是否已读
        for t in READ_TYPE:
            drop_carrier_df = data_drop_carrier[data_drop_carrier['read'] == t]  # 去除运营商的数据
            eng_sender_df = data_eng_sender[data_eng_sender['read'] == t]  # sender为字母的数据
            carrier_df = data_carrier[data_carrier['read'] == t]  # 运营商的数据
            dig_sender_df = data_dig_sender[data_dig_sender['read'] == t]  # sender为数字的数据
            all_df = data[data['read'] == t]
            sms_feature_res = gen_cnt_fea(all_df, drop_carrier_df, 'drop_carrier_read_' + t, sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, eng_sender_df, 'eng_sender_read_' + t, sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, carrier_df, 'carrier_read_' + t, sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, dig_sender_df, 'dig_sender_read_' + t, sms_feature_res)

            all_cnt = calc_cnt_fea(all_df)
            for fea in all_cnt:
                sms_feature_res[f"{fea}_read_{t}"] = all_cnt[fea]
                sms_feature_res[f"{fea}_rto_read_{t}"] = all_cnt[fea] / data_cnt[fea] if data_cnt[fea] > 0 else -999
        # 接收发送
        for t in SMS_TYPE:
            drop_carrier_df = data_drop_carrier[data_drop_carrier['type'] == t]  # 去除运营商的数据
            eng_sender_df = data_eng_sender[data_eng_sender['type'] == t]  # sender为字母的数据
            carrier_df = data_carrier[data_carrier['type'] == t]  # 运营商的数据
            dig_sender_df = data_dig_sender[data_dig_sender['type'] == t]  # sender为数字的数据
            all_df = data[data['type'] == t]
            sms_feature_res = gen_cnt_fea(all_df, drop_carrier_df, f'drop_carrier_{SMS_TYPE[t]}', sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, eng_sender_df, f'eng_sender_{SMS_TYPE[t]}', sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, carrier_df, f'carrier_{SMS_TYPE[t]}', sms_feature_res)
            sms_feature_res = gen_cnt_fea(all_df, dig_sender_df, f'dig_sender_{SMS_TYPE[t]}', sms_feature_res)

            all_cnt = calc_cnt_fea(all_df)
            for fea in all_cnt:
                sms_feature_res[f"{fea}_{SMS_TYPE[t]}"] = all_cnt[fea]
                sms_feature_res[f"{fea}_rto_{SMS_TYPE[t]}"] = all_cnt[fea] / data_cnt[fea] if data_cnt[fea] > 0 else -999

        # 去掉占比最大的sender
        top_types = ['top1', 'top3']
        if data.shape[0] != 0:

            for top_type in top_types:
                if top_type == 'top1':
                    top_sender = data['sender'].value_counts().sort_index().idxmax()
                    drop_top_sender_df = data[data['sender'] != top_sender]
                else:
                    tmp_sort = data['sender'].value_counts().reset_index()
                    tmp_sort.columns = ['sender', 'cnt']
                    top_sender = set(
                        tmp_sort.sort_values(by=['cnt', 'sender'],ascending=False).head(3)['sender'].values)
                    drop_top_sender_df = data[data['sender'].apply(lambda x: x not in top_sender)]
                all_cnt = calc_cnt_fea(drop_top_sender_df)
                for fea in all_cnt:
                    sms_feature_res[f"{fea}_drop_{top_type}_sender"] = all_cnt[fea]

                # top1 sender 短信占比
                sms_feature_res[f'{top_type}_sender_body_rto'] = 1 - drop_top_sender_df.shape[0] / data.shape[0]
        else:
            for top_type in top_types:
                for fea in FEA_LST:
                    sms_feature_res[f"{fea}_drop_{top_type}_sender"] = -999
                sms_feature_res[f'{top_type}_sender_body_rto'] = -999

        return sms_feature_res

    @staticmethod
    def extract_similar_paris_feature(data, conf):
        sms_feature_res = {}
        data_drop_carrier = data[data['is_carrier'] == 0]
        df_types = {'_drop_carrier': data_drop_carrier, '': data}

        for df_type in df_types:
            try:
                documents = df_types[df_type]['body'].values
                # 初始化TF-IDF向量化器
                vectorizer = TfidfVectorizer(stop_words=conf["stopwords"])
                # 将文本转换为TF-IDF特征矩阵
                tfidf_matrix = vectorizer.fit_transform(documents)
                # 计算TF-IDF特征矩阵的余弦相似度
                cosine_similarities = cosine_similarity(tfidf_matrix)
                # 定义余弦相似度阈值
                threshold = 0.5
                # 初始化计数器
                similar_text_count = 0
                # 遍历余弦相似度矩阵，跳过对角线元素（自身比较）
                for i in range(len(cosine_similarities)):
                    for j in range(i + 1, len(cosine_similarities)):
                        if cosine_similarities[i, j] >= threshold:
                            similar_text_count += 1
                similar_text_count /= 2
                sms_feature_res[f'similar_paris{df_type}'] = math.ceil(similar_text_count)
            except:
                sms_feature_res[f'similar_paris{df_type}'] = -999
        return sms_feature_res

    @staticmethod
    def extract_top_feature(data):
        # 运营商/非运营商、字母sender/非字母sender
        sms_feature_res = {}
        top_types = {'sender': 'is_digit', 'carrier': 'is_carrier'}
        for t in TIME_INTERVALS:
            if t == 'all':
                all_df = data.copy()
            else:
                all_df = data[data['time_diff'] < t]
                t = 'd' + str(t)
            for top_type in top_types:
                tmp_res = gen_top_fea(all_df.explode('word'), top_types[top_type], FEA_LST)
                for fea in tmp_res:
                    fea_name = fea.replace('type', top_type)
                    sms_feature_res[f'{fea_name}_{t}'] = tmp_res[fea]
                # 计算每个类别的max sender
                key_fea = top_types[top_type]
                tmp_df1 = all_df[all_df[key_fea] == 1]
                tmp_df0 = all_df[all_df[key_fea] == 0]
                tmp_res1 = gen_top_fea(tmp_df1.explode('word'), 'sender', FEA_LST)
                tmp_res0 = gen_top_fea(tmp_df0.explode('word'), 'sender', FEA_LST)
                for fea in tmp_res1:
                    fea_name1 = fea.replace('type', 'sender') + '_' + key_fea
                    sms_feature_res[f'{fea_name1}_{t}'] = tmp_res1[fea]
                for fea in tmp_res0:
                    fea_name0 = fea.replace('type', 'sender') + '_' + key_fea.replace('is', 'not')
                    sms_feature_res[f'{fea_name0}_{t}'] = tmp_res0[fea]

        return sms_feature_res

    @staticmethod
    def extract_day_feature(data):
        """
        计算按天聚合的相关特征
        :return:
        """
        sms_feature_res = {}
        time_type = 'time_diff'
        data_drop_carrier = data[data['is_carrier'] == 0]
        data_eng_sender = data[data['is_digit'] == 0]
        data_carrier = data[data['is_carrier'] == 1]
        data_dig_sender = data[data['is_digit'] == 1]

        # 联系密度
        df_types = {'drop_carrier_': data_drop_carrier, 'carrier_': data_carrier,
                    'eng_sender_': data_eng_sender, 'dig_sender_': data_dig_sender}
        for t in TIME_INTERVALS:
            if t != 'all':
                for df_type in df_types:
                    tmp_df = df_types[df_type].copy()
                    tmp_df = tmp_df[tmp_df[time_type] < t]
                    density = calc_density(tmp_df, t)
                    sms_feature_res[f'density_{df_type}d{t}'] = density

        # 计算聚合天数、连续空档数和连续有值数等相关特征
        for t in TIME_INTERVALS:
            if t == 'all':
                drop_carrier_df = data_drop_carrier.copy()  # 去除运营商的数据
                carrier_df = data_carrier.copy()  # 运营商数据
                eng_sender_df = data_eng_sender.copy()  # sender为字母的数据
                dig_sender_df = data_dig_sender.copy()  # sender为数字的数据
                all_df = data.copy()
            else:
                drop_carrier_df = data_drop_carrier[data_drop_carrier[time_type] < t]  # 去除运营商的数据
                carrier_df = data_carrier[data_carrier[time_type] < t]  # 运营商数据
                eng_sender_df = data_eng_sender[data_eng_sender[time_type] < t]  # sender为字母的数据
                dig_sender_df = data_dig_sender[data_dig_sender[time_type] < t]  # sender为字母的数据
                all_df = data[data[time_type] < t]
                t = 'd' + str(t)

            # 聚合之后，每一天的 cnt的 max/min/avg/sd
            df_types = {'drop_carrier_': drop_carrier_df, 'carrier_': carrier_df, 'eng_sender_': eng_sender_df,
                        'dig_sender_': dig_sender_df, '': all_df}
            for df_type in df_types:
                tmp_df = df_types[df_type].copy()
                res_agg = calc_agg_fea(tmp_df)
                for i in res_agg:
                    sms_feature_res[f'{i}_{df_type}{t}'] = res_agg[i]

                # 计算连续和空档相关特征
                res_con = calc_continue(tmp_df, t)
                for i in res_con:
                    if 'long' not in i:
                        sms_feature_res[f'{i}_{df_type}{t}'] = res_con[i]

        return sms_feature_res

    @staticmethod
    def extract_se_feature(data):
        """
        se(开始结束)模块:
        :return:
        """
        sms_feature_res = {}
        data_drop_carrier = data[data['is_carrier'] == 0]
        data_eng_sender = data[data['is_digit'] == 0]
        data_carrier = data[data['is_carrier'] == 1]
        data_dig_sender = data[data['is_digit'] == 1]

        df_types = {'_drop_carrier': data_drop_carrier, '_carrier': data_carrier,
                    '_eng_sender': data_eng_sender, '_dig_sender': data_dig_sender, '': data}
        for df_type in df_types:
            tmp_df_all = df_types[df_type].copy()
            # 计算se、ae、as特征
            res_se = calc_se_fea(tmp_df_all)
            for se_type in SE_TYPE:
                sms_feature_res[f'{se_type}_days_diff{df_type}'] = res_se[se_type]
            # 计算cnt
            res_cnt = calc_cnt_fea(tmp_df_all)
            # 计算cnt/ 开始 - 结束 天数差
            for fea in FEA_LST:
                cnt = res_cnt[fea]
                se = sms_feature_res[f'es_days_diff{df_type}']
                sms_feature_res[f'{fea}_to_es_days_diff{df_type}_rto'] = cnt / se if se > 0 else -999

            # 计算去掉空档的se相关特征
            if tmp_df_all.shape[0] != 0:
                tmp_day = tmp_df_all['time_day'].unique()
                sms_time_lst = [datetime.strptime(i, '%Y-%m-%d') for i in sorted(tmp_day)]
                first_date_drop30 = sms_time_lst[0]
                first_date_drop60 = sms_time_lst[0]
                for i in range(1, len(sms_time_lst)):
                    diff_day = (sms_time_lst[i] - sms_time_lst[i - 1]).days - 1
                    if diff_day >= 60:
                        first_date_drop60 = sms_time_lst[i]
                    if diff_day >= 30:
                        first_date_drop30 = sms_time_lst[i]

                drop_types = {'drop30': str(first_date_drop30)[:10], 'drop60': str(first_date_drop60)[:10]}
                for drop in drop_types:
                    tmp_df = tmp_df_all[tmp_df_all['time_day'] >= drop_types[drop]]
                    # 计算se、ae、as特征
                    tmp_res_se = calc_se_fea(tmp_df)
                    for se_type in SE_TYPE:
                        sms_feature_res[f'{se_type}_days_diff{df_type}_{drop}'] = tmp_res_se[se_type]
                    # 计算cnt
                    tmp_res_cnt = calc_cnt_fea(tmp_df)
                    # 计算cnt/ 开始 - 结束 天数差
                    for fea in FEA_LST:
                        cnt = tmp_res_cnt[fea]
                        se = sms_feature_res[f'es_days_diff{df_type}_{drop}']
                        sms_feature_res[f'{fea}_to_es_days_diff{df_type}_rto_{drop}'] = cnt / se if se > 0 else -999
            else:
                for drop in ['drop30', 'drop60']:
                    # 计算se、ae、as特征
                    for se_type in SE_TYPE:
                        sms_feature_res[f'{se_type}_days_diff{df_type}_{drop}'] = -999
                    # 计算cnt/ 开始 - 结束 天数差
                    for fea in FEA_LST:
                        sms_feature_res[f'{fea}_to_es_days_diff{df_type}_rto_{drop}'] = -999

        return sms_feature_res

    @staticmethod
    def extract_dod_feature(data):
        """"
        计算环比和环差
        """
        sms_feature_res = {}
        data_drop_carrier = data[data['is_carrier'] == 0]
        data_eng_sender = data[data['is_digit'] == 0]
        data_carrier = data[data['is_carrier'] == 1]
        data_dig_sender = data[data['is_digit'] == 1]

        df_types = {'_drop_carrier': data_drop_carrier, '_carrier': data_carrier,
                    '_eng_sender': data_eng_sender, '_dig_sender': data_dig_sender, '': data}

        for df_type in df_types:
            for i in WINDOW_LST:
                for fea in FEA_LST:
                    if fea == 'word':
                        tmp_df = df_types[df_type].explode('word')
                    else:
                        tmp_df = df_types[df_type].copy()
                    res = {}
                    res = calc_dod_cnt_rto(tmp_df, i, fea, res)
                    for fea_name in res:
                        sms_feature_res[f'{fea_name}{df_type}'] = res[fea_name]

        return sms_feature_res
